#!/usr/bin/env python

import re

class KalTokenizer(object):
    escape_map = {'n':'\n','r':'\r','t':'\t'}
    WHITESPACE = re.compile(r'([ \t\r\f\v]+(#.*$)?)|(#.*$)', re.MULTILINE)
    re_map = {'INTEGER':     re.compile(r'([0-9]+)|(0x[0-9a-fA-F]+)',re.MULTILINE),
              'FLOAT':       re.compile(r'([0-9]+\.[0-9]*)|(\.[0-9]+)',re.MULTILINE),
              'STRING':      re.compile(r'(\"(\\.|[^\"])*\")|' + r"(\'(\\.|[^\'])*\')",re.MULTILINE),
              'NAME':        re.compile(r'[A-Za-z_][A-Za-z_0-9]*',re.MULTILINE),
              'BINOP':       re.compile(r'[><=\!]=|[\+\-\*\/\^\&\|\%><]',re.MULTILINE),
              'PUNCTUATION': re.compile(r'\S',re.MULTILINE),
              'NEWLINE':     re.compile(r'\n',re.MULTILINE)}
    def __init__(self,kal_code):
        self.kal_code = kal_code
        self.char_index = 0
        self.line_no = 1
        self.column_no = 1
        self.token_type = 'UNKNOWN'
        self.token = ''
        self.stack = []
    def push(self):
        self.stack.append((self.char_index,self.line_no,self.column_no,self.token_type,self.token))
    def pop(self):
        (self.char_index,self.line_no,self.column_no,self.token_type,self.token) = self.stack.pop()
    def __iter__(self):
        return self
    def eat_whitespace(self):
        m = self.WHITESPACE.match(self.kal_code,self.char_index)
        if m:
            self.char_index = m.end()
            self.column_no += m.end() - m.start()
    def unescape(self,s):
        escape = False
        rv = ""
        for c in s:
            if escape:
                if c in self.escape_map:
                    rv += self.escape_map[c]
                else:
                    rv += c
            elif c == '\\':
                escape = True
            else:
                rv += c
        return rv
    def next(self):
        if self.token_type == 'NEWLINE':
            self.line_no += 1
            self.column_no = 1
        else:
            self.column_no += len(self.token)
        self.eat_whitespace()
        if self.char_index >= len(self.kal_code):
            self.token_type = 'EOF'
            self.token = ''
            return ''
        for token_type in ('FLOAT','INTEGER','STRING','NAME','NEWLINE','BINOP','PUNCTUATION'):
            m = self.re_map[token_type].match(self.kal_code,self.char_index)
            if m:
                self.token_type = token_type
                self.token = self.kal_code[m.start():m.end()]
                self.char_index = m.end()
                return self.token
        print self.char_index, ">" + self.kal_code[self.char_index:]
        raise SyntaxError, "Invalid token: " + self.kal_code[self.char_index:self.char_index+10] + "... on line " + str(self.line_no) + " at column " + str(self.column_no)
    def check_token(self,token):
        if token != self.token:
            raise SyntaxError, "Unexpected: " + token + " on line " + str(self.line_no) + " at column " + str(self.column_no)
    def require_token(self,token):
        self.next()
        self.check_token(token)
    def check_type(self,token_type):
        if token_type != self.token_type:
            raise SyntaxError, "Unexpected: " + token + " on line " + str(self.line_no) + " at column " + str(self.column_no)
    def require_type(self,token_type):
        self.next()
        self.check_type(token_type)
    def set_info(self,obj):
        obj.line_no = self.line_no
        obj.column_no = self.column_no
